{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get all imports and silence those pesky warninigs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from lightgbm import LGBMRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from catboost import CatBoostRegressor\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Brand</th>\n",
       "      <th>Model_Info</th>\n",
       "      <th>Locality</th>\n",
       "      <th>City</th>\n",
       "      <th>State</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 name234 64gb space grey</td>\n",
       "      <td>878</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>15000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>phone 7 name42 name453 new condition box acces...</td>\n",
       "      <td>1081</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>18800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 x 256gb leess used good condition</td>\n",
       "      <td>495</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 6s plus 64 gb space grey</td>\n",
       "      <td>287</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>16500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>phone 7 sealed pack brand new factory outet price</td>\n",
       "      <td>342</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>26499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Brand                                         Model_Info  Locality  City  \\\n",
       "0      1                      name0 name234 64gb space grey       878     8   \n",
       "1      1  phone 7 name42 name453 new condition box acces...      1081     4   \n",
       "2      1            name0 x 256gb leess used good condition       495    11   \n",
       "3      1                     name0 6s plus 64 gb space grey       287    10   \n",
       "4      1  phone 7 sealed pack brand new factory outet price       342     4   \n",
       "\n",
       "   State  Price  \n",
       "0      2  15000  \n",
       "1      0  18800  \n",
       "2      4  50000  \n",
       "3      7  16500  \n",
       "4      0  26499  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"Train.csv\")\n",
    "df_test = pd.read_csv(\"Test.csv\")\n",
    "\n",
    "# This column is totally useless\n",
    "df_train.drop([\"Additional_Description\"], axis=1, inplace=True)\n",
    "df_test.drop([\"Additional_Description\"], axis=1, inplace=True)\n",
    "\n",
    "# All cleaning was done on the csv file itself\n",
    "# The model_info column contains a white-space at the beginning\n",
    "df_train['Model_Info'] = df_train['Model_Info'].str.strip()\n",
    "df_test['Model_Info'] = df_test['Model_Info'].str.strip()\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions to create new features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0 honor \n",
    "# 1 iphone and iwatch\n",
    "# 2 lenovo\n",
    "# 3 lg\n",
    "\n",
    "\n",
    "# Load the un-normalized sum of idf weighted vectors\n",
    "# for both train and test sentences of the model_info column\n",
    "def load_sent_vecs(path):\n",
    "    fp = open(path, \"rb\")\n",
    "    sent_vecs = pickle.load(fp)\n",
    "    fp.close()\n",
    "    return sent_vecs\n",
    "\n",
    "\n",
    "# Check if phone is not in a good condition\n",
    "# The words were extracted from what I observed in the dataset\n",
    "def get_bad_condition(model_info):\n",
    "    cond = list()\n",
    "    for index, text in enumerate(model_info):\n",
    "        if \"dead\" in text or \"crack\" in text:\n",
    "            cond.append(1)\n",
    "        else:\n",
    "            cond.append(0)\n",
    "    return cond\n",
    "\n",
    "\n",
    "# Helper function to extract number for usage in ram and rom features\n",
    "# Either the current word contains ram/rom data\n",
    "# Or the previous word, e.g. 23 gb (previous word) or 23gb (current word)\n",
    "def extract_num(text1, text2):\n",
    "    # So rom is either current or previous\n",
    "    try:\n",
    "        space = int(''.join(filter(str.isdigit, text2)))\n",
    "    except:\n",
    "        space = int(''.join(filter(str.isdigit, text1)))\n",
    "    return space\n",
    "\n",
    "\n",
    "# Creates the rom feature\n",
    "def get_rom(model_info):\n",
    "    rom = list()\n",
    "    for index, text in enumerate(model_info):\n",
    "        space, spaces = \"\", list()\n",
    "        for i in range(len(text)):\n",
    "            # If either of these two keywords are present\n",
    "            # then it means that it is either ram or rom\n",
    "            if \"gb\" in text[i] or \"gig\" in text[i]:\n",
    "                # Make sure \"ram\" is not to the right\n",
    "                if (i < len(text)-1 and \"ram\" not in text[i+1]) or i == len(text)-1:\n",
    "                    space = extract_num(text[i-1], text[i])\n",
    "                # The current word could also be the first word\n",
    "                # In that case, no number could be present before the current word\n",
    "                elif i == 0:\n",
    "                    space = extract_num(None, text[i])\n",
    "                # Using a list as there could be two numbers\n",
    "                # each representing ram and rom\n",
    "                if type(space) == int:\n",
    "                    spaces.append(space)\n",
    "        # Only the largest number is rom.\n",
    "        if len(spaces) > 0:\n",
    "            rom.append(max(spaces))\n",
    "        else:\n",
    "            rom.append(0)\n",
    "    return rom\n",
    "\n",
    "\n",
    "# Creates the ram feature\n",
    "def get_ram(model_info):\n",
    "    ram = list()\n",
    "    for index, text in enumerate(model_info):\n",
    "        space, spaces, its_ram = \"\", list(), False\n",
    "        for i in range(len(text)):\n",
    "            # If either of these two keywords are present\n",
    "            # then it means that it is either ram or rom\n",
    "            if \"gb\" in text[i] or \"gig\" in text[i]:\n",
    "                # If \"ram\" is to the right then term is definitely ram\n",
    "                if i < len(text)-1 and \"ram\" in text[i+1]:\n",
    "                    space = extract_num(text[i-1], text[i])\n",
    "                    its_ram = True\n",
    "                # In some cases, \"ram\" is not written\n",
    "                elif i < len(text)-1 or i == len(text)-1:\n",
    "                    space = extract_num(text[i-1], text[i])\n",
    "                # The current word could also be the first word\n",
    "                # In that case, no number could be present before the current word\n",
    "                elif i == 0:\n",
    "                    space = extract_num(None, text[i])\n",
    "                if type(space) == int:\n",
    "                    spaces.append(space)\n",
    "        # If there is only one number and its_ram is true then it is definitely ram\n",
    "        if its_ram == True and len(spaces) <= 1:\n",
    "            ram.append(space)\n",
    "        # If there are multiple numbers, then the minimum is ram\n",
    "        elif len(spaces) > 1:\n",
    "            ram.append(min(spaces))\n",
    "        else:\n",
    "            ram.append(0)\n",
    "    return ram\n",
    "\n",
    "\n",
    "# Check if phone has warranty\n",
    "# The word were extracted from what I observed in the dataset\n",
    "def get_warranty(model_info):\n",
    "    warranty = list()\n",
    "    for text in model_info:\n",
    "        if \"war\" in text:\n",
    "            warranty.append(1)\n",
    "        else:\n",
    "            warranty.append(0)\n",
    "    return warranty\n",
    "\n",
    "\n",
    "# Check if payment needs to be made in terms of cash or online\n",
    "# The words were extracted from what I observed in the dataset\n",
    "def get_cash(model_info):\n",
    "    cash = list()\n",
    "    for text in model_info:\n",
    "        if \"cas\" in text:\n",
    "            cash.append(1)\n",
    "        else:\n",
    "            cash.append(0)\n",
    "    return cash\n",
    "\n",
    "\n",
    "# This function gives two features -> iphone and iwatch\n",
    "# The iphone features is the most important out of all my engineered features\n",
    "# Higher price (in the real world) iphone/iwatch will have higher number\n",
    "# The order of the if-else statements is crucial\n",
    "# as it tells you about the priority of the products\n",
    "def get_apple_product(brand_type, model_info):\n",
    "    iphone_type, iwatch_type = list(), list()\n",
    "    for brand, text in zip(brand_type, model_info):\n",
    "        if brand == 1:\n",
    "            if \"watch\" in text or \"iwatch\" in text:\n",
    "                iphone_type.append(0)\n",
    "                if \"5\" in text:\n",
    "                    iwatch_type.append(5)\n",
    "                elif \"4\" in text:\n",
    "                    iwatch_type.append(4)\n",
    "                elif \"3\" in text:\n",
    "                    iwatch_type.append(3)\n",
    "                elif \"2\" in text:\n",
    "                    iwatch_type.append(2)\n",
    "                elif \"1\" in text:\n",
    "                    iwatch_type.append(1)\n",
    "                else:\n",
    "                    iwatch_type.append(1)\n",
    "            else:\n",
    "                iwatch_type.append(0)\n",
    "                if (\"11\" in text or \"eleven\" in text or \"elven\" in text) and \"pro\" in text and \"max\" in text:\n",
    "                    iphone_type.append(29)\n",
    "                elif (\"11\" in text or \"eleven\" in text or \"elven\" in text) and \"pro\" in text:\n",
    "                    iphone_type.append(28)\n",
    "                elif \"11\" in text or \"eleven\" in text or \"elven\" in text:\n",
    "                    iphone_type.append(27)\n",
    "                elif \"xs\" in text and \"max\" in text:\n",
    "                    iphone_type.append(26)\n",
    "                elif \"xs\" in text:\n",
    "                    iphone_type.append(25)\n",
    "                elif \"x\" in text:\n",
    "                    iphone_type.append(24)\n",
    "                elif (\"8s\" in text and \"plus\" in text) or (\"8\" in text and \"s\" in text and \"plus\" in text) or (\"8\" in text and \"splus\" in text) or \"8splus\" in text:\n",
    "                    iphone_type.append(23)\n",
    "                elif (\"8\" in text and \"plus\" in text) or \"8plus\" in text:\n",
    "                    iphone_type.append(22)\n",
    "                elif (\"8\" in text and \"s\" in text) or \"8s\" in text:\n",
    "                    iphone_type.append(21)\n",
    "                elif \"8\" in text:\n",
    "                    try:\n",
    "                        next_string = text[text.index(\"8\")+1]\n",
    "                        if \"month\" not in next_string or \"year\" not in next_string or \"time\" not in next_string:\n",
    "                            iphone_type.append(20)\n",
    "                    except:\n",
    "                        iphone_type.append(20)\n",
    "                elif (\"7s\" in text and \"plus\" in text) or (\"7\" in text and \"s\" in text and \"plus\" in text) or (\"7\" in text and \"splus\" in text) or \"7splus\" in text:\n",
    "                    iphone_type.append(19)\n",
    "                elif (\"7\" in text and \"plus\" in text) or \"7plus\" in text:\n",
    "                    iphone_type.append(18)\n",
    "                elif (\"7\" in text and \"s\" in text) or \"7s\" in text:\n",
    "                    iphone_type.append(17)\n",
    "                elif \"7\" in text:\n",
    "                    try:\n",
    "                        next_string = text[text.index(\"7\")+1]\n",
    "                        if \"month\" not in next_string or \"year\" not in next_string or \"time\" not in next_string:\n",
    "                            iphone_type.append(16)\n",
    "                    except:\n",
    "                        iphone_type.append(16)\n",
    "                elif (\"6s\" in text and \"plus\" in text) or (\"6\" in text and \"s\" in text and \"plus\" in text) or (\"6\" in text and \"splus\" in text) or \"6splus\" in text:\n",
    "                    iphone_type.append(15)\n",
    "                elif (\"6\" in text and \"plus\" in text) or \"6plus\" in text:\n",
    "                    iphone_type.append(14)\n",
    "                elif (\"6\" in text and \"s\" in text) or \"6s\" in text:\n",
    "                    iphone_type.append(13)\n",
    "                elif \"6\" in text:\n",
    "                    try:\n",
    "                        next_string = text[text.index(\"6\")+1]\n",
    "                        if \"month\" not in next_string or \"year\" not in next_string or \"time\" not in next_string:\n",
    "                            iphone_type.append(12)\n",
    "                    except:\n",
    "                        iphone_type.append(12)\n",
    "                elif (\"5s\" in text and \"plus\" in text) or (\"5\" in text and \"s\" in text and \"plus\" in text) or (\"5\" in text and \"splus\" in text) or \"5splus\" in text:\n",
    "                    iphone_type.append(11)\n",
    "                elif (\"5\" in text and \"plus\" in text) or \"5plus\" in text:\n",
    "                    iphone_type.append(10)\n",
    "                elif (\"5\" in text and \"s\" in text) or \"5s\" in text:\n",
    "                    iphone_type.append(9)\n",
    "                elif (\"5\" in text and \"c\" in text) or \"5c\" in text:\n",
    "                    iphone_type.append(8)\n",
    "                elif \"5\" in text:\n",
    "                    try:\n",
    "                        next_string = text[text.index(\"5\")+1]\n",
    "                        if \"month\" not in next_string or \"year\" not in next_string or \"time\" not in next_string:\n",
    "                            iphone_type.append(7)\n",
    "                    except:\n",
    "                        iphone_type.append(7)\n",
    "                elif (\"4s\" in text and \"plus\" in text) or (\"4\" in text and \"s\" in text and \"plus\" in text) or (\"4\" in text and \"splus\" in text) or \"4splus\" in text:\n",
    "                    iphone_type.append(6)\n",
    "                elif (\"4\" in text and \"plus\" in text) or \"4plus\" in text:\n",
    "                    iphone_type.append(5)\n",
    "                elif (\"4\" in text and \"s\" in text) or \"4s\" in text:\n",
    "                    iphone_type.append(4)\n",
    "                elif (\"4\" in text and \"c\" in text) or \"4c\" in text:\n",
    "                    iphone_type.append(3)\n",
    "                elif \"4\" in text:\n",
    "                    try:\n",
    "                        next_string = text[text.index(\"4\")+1]\n",
    "                        if \"month\" not in next_string or \"year\" not in next_string or \"time\" not in next_string:\n",
    "                            iphone_type.append(2)\n",
    "                    except:\n",
    "                        iphone_type.append(2)\n",
    "                else:\n",
    "                    iphone_type.append(1)\n",
    "        else:\n",
    "            iphone_type.append(0)\n",
    "            iwatch_type.append(0)\n",
    "    return iphone_type, iwatch_type\n",
    "\n",
    "\n",
    "# Gives the same feature as iphone above but for LG\n",
    "def get_lg_product(brand_type, model_info):\n",
    "    lg_type= list()\n",
    "    for brand, text in zip(brand_type, model_info):\n",
    "        if brand == 3:\n",
    "            if (\"v\" in text and \"50\" in text) or \"v50\" in text:\n",
    "                lg_type.append(25)\n",
    "            elif (\"v\" in text and \"40\" in text) or \"v40\" in text:\n",
    "                lg_type.append(24)\n",
    "            elif (\"g\" in text and \"7\" in text) or \"g7\" in text or \"7\" in text:\n",
    "                lg_type.append(23)\n",
    "            elif (\"v\" in text and \"30\" in text) or \"v30\" in text:\n",
    "                lg_type.append(22)\n",
    "            elif (\"v\" in text and \"20\" in text) or \"v20\" in text:\n",
    "                lg_type.append(21)\n",
    "            elif (\"nexus\" in text and \"5\" in text and \"x\" in text) or (\"nexus\" in text and \"5x\" in text) or (\"nexus5\" in text and \"x\" in text) or \"nexus5x\" in text:\n",
    "                lg_type.append(20)\n",
    "            elif (\"g\" in text and \"6\" in text) or \"g6\" in text or \"6\" in text:\n",
    "                lg_type.append(19)  \n",
    "            elif (\"thin\" in text and \"q\" in text) or \"thinq\" in text:\n",
    "                lg_type.append(18)\n",
    "            elif \"plus\" in text:\n",
    "                lg_type.append(17)\n",
    "            elif \"nexus\" in text:\n",
    "                lg_type.append(16)\n",
    "            elif (\"stylush\" in text and \"2\" in text) or \"stylush2\" in text:\n",
    "                lg_type.append(15)\n",
    "            elif \"stylush\" in text:\n",
    "                lg_type.append(14)\n",
    "            elif (\"g\" in text and \"4\" in text) or \"g4\" in text or \"4\" in text:\n",
    "                lg_type.append(13)\n",
    "            elif (\"q\" in text and \"7\" in text and \"plus\" in text) or (\"q7\" in text and \"plus\" in text) or (\"q\" in text and \"7plus\" in text) or \"q7plus\" in text:\n",
    "                lg_type.append(12)\n",
    "            elif (\"g\" in text and \"5\" in text) or \"g5\" in text or \"5\" in text:\n",
    "                lg_type.append(11)\n",
    "            elif (\"q\" in text and \"6\" in text) or \"q6\" in text:\n",
    "                lg_type.append(10)\n",
    "            elif \"2017\" in text:\n",
    "                lg_type.append(9)\n",
    "            elif (\"q\" in text and \"7\" in text) or \"q7\" in text:\n",
    "                lg_type.append(8)\n",
    "            elif \"4g\" in text and \"volte\" in text and \"dual\" in text:\n",
    "                lg_type.append(7)\n",
    "            elif (\"g\" in text and \"3\" in text) or \"g3\" in text:\n",
    "                lg_type.append(6)\n",
    "            elif (\"w\" in text and \"30\" in text) or \"w30\" in text:\n",
    "                lg_type.append(5)\n",
    "            elif (\"k\" in text and \"10\" in text) or \"k10\" in text:\n",
    "                lg_type.append(4)\n",
    "            elif \"2\" in text or \"g2\" in text or (\"g\" in text and \"2\" in text):\n",
    "                lg_type.append(3)\n",
    "            elif \"ph2\" in text:\n",
    "                lg_type.append(2)\n",
    "            else:\n",
    "                lg_type.append(1)\n",
    "        else:\n",
    "            lg_type.append(0)\n",
    "    return lg_type\n",
    "\n",
    "\n",
    "# Gives the same feature as iphone above but for Huawei Honor\n",
    "def get_honor_product(brand_type, model_info):\n",
    "    honor_type= list()\n",
    "    for brand, text in zip(brand_type, model_info):\n",
    "        if brand == 0:\n",
    "            if \"9\" in text:\n",
    "                honor_type.append(29)\n",
    "            elif \"porsche\" in text:\n",
    "                honor_type.append(28)\n",
    "            elif (\"view\" in text and \"20\" in text) or \"view20\" in text:\n",
    "                honor_type.append(27)\n",
    "            elif \"10\" in text:\n",
    "                honor_type.append(26)\n",
    "            elif (\"view\" in text and \"10\" in text) or \"view10\" in text:\n",
    "                honor_type.append(25)\n",
    "            elif (\"8\" in text and \"pro\" in text) or \"8pro\" in text:\n",
    "                honor_type.append(24)\n",
    "            elif (\"nova\" in text and \"2\" in text and \"plus\" in text) or (\"nova\" in text and \"2plus\" in text) or (\"nova2\" in text and \"plus\" in text) or \"nova2plus\" in text:\n",
    "                honor_type.append(23)\n",
    "            elif \"20\" in text:\n",
    "                honor_type.append(22)\n",
    "            elif (\"nova\" in text and \"3\" in text and \"i\" in text) or (\"nova\" in text and \"3i\" in text) or (\"nova3\" in text and \"i\" in text) or \"nova3i\" in text:\n",
    "                honor_type.append(21)\n",
    "            elif \"8\" in text:\n",
    "                honor_type.append(20)\n",
    "            elif (\"970\" in text and \"i\" in text) or \"970i\" in text:\n",
    "                honor_type.append(19)\n",
    "            elif (\"970\" in text and \"i\" in text) or \"970i\" in text:\n",
    "                honor_type.append(18)\n",
    "            elif (\"p\" in text and \"20\" in text and \"lite\" in text) or (\"p\" in text and \"20lite\" in text) or (\"p20\" in text and \"lite\" in text) or \"p20lite\" in text:\n",
    "                honor_type.append(17)\n",
    "            elif (\"8\" in text and \"x\" in text) or \"8x\" in text:\n",
    "                honor_type.append(16)\n",
    "            elif (\"7\" in text and \"x\" in text) or \"7x\" in text:\n",
    "                honor_type.append(15)\n",
    "            elif (\"10\" in text and \"lite\" in text) or \"10lite\" in text:\n",
    "                honor_type.append(14)\n",
    "            elif (\"6\" in text and \"x\" in text) or \"6x\" in text:\n",
    "                honor_type.append(13)\n",
    "            elif (\"8\" in text and \"lite\" in text) or \"8lite\" in text:\n",
    "                honor_type.append(12)\n",
    "            elif (\"9\" in text and \"lite\" in text) or \"9lite\" in text:\n",
    "                honor_type.append(11)\n",
    "            elif (\"9\" in text and \"n\" in text) or \"9n\" in text:\n",
    "                honor_type.append(10)\n",
    "            elif (\"7\" in text and \"c\" in text) or \"7c\" in text:\n",
    "                honor_type.append(9)\n",
    "            elif (\"8\" in text and \"c\" in text) or \"8c\" in text:\n",
    "                honor_type.append(8)\n",
    "            elif (\"5\" in text and \"x\" in text) or \"5x\" in text:\n",
    "                honor_type.append(7)\n",
    "            elif (\"7\" in text and \"a\" in text) or \"7a\" in text:\n",
    "                honor_type.append(6)\n",
    "            elif (\"holly\" in text and \"4\" in text and \"plus\" in text) or (\"holly\" in text and \"4plus\" in text) or (\"holly4\" in text and \"plus\" in text) or \"holly4plus\" in text:\n",
    "                honor_type.append(5)\n",
    "            elif (\"4\" in text and \"x\" in text) or \"4x\" in text:\n",
    "                honor_type.append(4)\n",
    "            elif \"3\" in text:\n",
    "                honor_type.append(3)\n",
    "            elif \"g520\" in text:\n",
    "                honor_type.append(2)\n",
    "            else:\n",
    "                honor_type.append(1)\n",
    "        else:\n",
    "            honor_type.append(0)\n",
    "    return honor_type\n",
    "\n",
    "\n",
    "# Gives the same feature as iphone above but for Lenovo\n",
    "def get_lenovo_product(brand_type, model_info):\n",
    "    lenovo_type= list()\n",
    "    for brand, text in zip(brand_type, model_info):\n",
    "        if brand == 2:\n",
    "            if \"k20\" in text:\n",
    "                lenovo_type.append(30)\n",
    "            elif \"x2 lenovo\" in text or \"x2\" in text:\n",
    "                lenovo_type.append(29)\n",
    "            elif \"lenovop2a42\" in text:\n",
    "                lenovo_type.append(28)\n",
    "            elif \" p1 \" in text:\n",
    "                lg_type.append(27)\n",
    "            elif \"zuk\" in text:\n",
    "                lenovo_type.append(26)\n",
    "            elif \"k6 note\" in text or (\"k6\" in text and \"note\" in text) or (\"k\" in text and \"6\" in text and \"note\" in text) or 'k6note' in text or 'notek6' in text:\n",
    "                lenovo_type.append(25)\n",
    "            elif \"a50\" in text:\n",
    "                lenovo_type.append(24)  \n",
    "            elif \"k4 note\" in text or (\"k4\" in text and \"note\" in text) or (\"k\" in text and \"4\" in text and \"note\" in text) or 'k4note' in text or 'notek4' in text:\n",
    "                lenovo_type.append(23)\n",
    "            elif \"vibe k5 note\" in text or (\"vibe\" in text and \"note\" in text and 'k5' in text):\n",
    "                lenovo_type.append(22)\n",
    "            elif \"k8 note\" in text or (\"k8\" in text and \"note\" in text) or (\"k\" in text and \"8\" in text and \"note\" in text) or 'k8note' in text or 'notek8' in text:\n",
    "                lenovo_type.append(21)\n",
    "            elif \"a20\" in text :\n",
    "                lenovo_type.append(20)\n",
    "            elif \"z2 plus\" in text or (\"z2\" in text and \"plus\" in text):\n",
    "                lenovo_type.append(19)\n",
    "            elif \"k5 note\" in text or (\"k5\" in text and \"note\" in text) or (\"k\" in text and \"5\" in text and \"note\" in text) or 'k5note' in text or 'notek5' in text:\n",
    "                lenovo_type.append(18)\n",
    "            elif \"k6 power\" in text or (\"k6\" in text and \"power\" in text) or (\"k\" in text and \"6\" in text and \"power\" in text) or 'k6power' in text:\n",
    "                lenovo_type.append(17)\n",
    "            elif \"name42tuxedo\" in text:\n",
    "                lenovo_type.append(16)\n",
    "            elif \"phab2plus\" in text or (\"phab\" in text and '2' in text and 'plus' in text) or (\"phab2\" in text and \"plus\" in text) or ('phab' in text and \"2plus\" in text):\n",
    "                lenovo_type.append(15)\n",
    "            elif \"vibeshot\" in text or (\"vibe\" in text and 'shot' in text):\n",
    "                lenovo_type.append(14)\n",
    "            elif \"k8plus\" in text or (\"k8\" in text and \"plus\" in text) or 'plusk8' in text or (\"k\" in text and \"8\" in text and \"plus\" in text) or (\"k\" in text and \"8plus\" in text):\n",
    "                lenovo_type.append(13)\n",
    "            elif \"k8\" in text or (\"k\" in text and \"8\" in text):\n",
    "                lenovo_type.append(12)\n",
    "            elif \"vibek5plus\" in text or \"k5plus\" in text or (\"k5\" in text and \"plus\" in text) or (\"vibe\" in text and \"k5\" in text and \"plus\" in text):\n",
    "                lenovo_type.append(11)\n",
    "            elif \"13 mpl back camera 8 mpl front cam\" in text:\n",
    "                lenovo_type.append(10)\n",
    "            elif \"a6600plus\" in text:\n",
    "                lenovo_type.append(9)\n",
    "            elif \"z1\" in text:\n",
    "                lenovo_type.append(8)\n",
    "            elif \"k3 note\" in text or (\"k3\" in text and \"note\" in text) or 'k3note' in text or 'notek3' in text:\n",
    "                lenovo_type.append(7)\n",
    "            elif \"p1m40\" in text:\n",
    "                lenovo_type.append(6)\n",
    "            elif \"p1m40\" in text or \"vibe p1m\" in text:\n",
    "                lenovo_type.append(5)\n",
    "            elif \"k5\" in text or \"five\" in text:\n",
    "                lenovo_type.append(4)\n",
    "            elif \"plus\" in text:\n",
    "                lenovo_type.append(3)\n",
    "            elif \"a6000\" in text:\n",
    "                lenovo_type.append(2)\n",
    "            else:\n",
    "                lenovo_type.append(1)\n",
    "        else:\n",
    "            lenovo_type.append(0)\n",
    "    return lenovo_type"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create new features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Brand</th>\n",
       "      <th>Model_Info</th>\n",
       "      <th>Locality</th>\n",
       "      <th>City</th>\n",
       "      <th>State</th>\n",
       "      <th>Price</th>\n",
       "      <th>Rom</th>\n",
       "      <th>Ram</th>\n",
       "      <th>Warranty</th>\n",
       "      <th>Cash</th>\n",
       "      <th>iphone_type</th>\n",
       "      <th>iwatch_type</th>\n",
       "      <th>LG_type</th>\n",
       "      <th>Honor_type</th>\n",
       "      <th>Lenovo_type</th>\n",
       "      <th>Bad_condition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 name234 64gb space grey</td>\n",
       "      <td>878</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>15000</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>phone 7 name42 name453 new condition box acces...</td>\n",
       "      <td>1081</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>18800</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 x 256gb leess used good condition</td>\n",
       "      <td>495</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>50000</td>\n",
       "      <td>256</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>name0 6s plus 64 gb space grey</td>\n",
       "      <td>287</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>16500</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>phone 7 sealed pack brand new factory outet price</td>\n",
       "      <td>342</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>26499</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Brand                                         Model_Info  Locality  City  \\\n",
       "0      1                      name0 name234 64gb space grey       878     8   \n",
       "1      1  phone 7 name42 name453 new condition box acces...      1081     4   \n",
       "2      1            name0 x 256gb leess used good condition       495    11   \n",
       "3      1                     name0 6s plus 64 gb space grey       287    10   \n",
       "4      1  phone 7 sealed pack brand new factory outet price       342     4   \n",
       "\n",
       "   State  Price  Rom  Ram  Warranty  Cash  iphone_type  iwatch_type  LG_type  \\\n",
       "0      2  15000   64    0         0     0            1            0        0   \n",
       "1      0  18800    0    0         0     0           16            0        0   \n",
       "2      4  50000  256    0         0     0           24            0        0   \n",
       "3      7  16500   64    0         0     0           15            0        0   \n",
       "4      0  26499    0    0         0     0           16            0        0   \n",
       "\n",
       "   Honor_type  Lenovo_type  Bad_condition  \n",
       "0           0            0              0  \n",
       "1           0            0              0  \n",
       "2           0            0              0  \n",
       "3           0            0              0  \n",
       "4           0            0              0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"Rom\"] = get_rom([i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"Rom\"] = get_rom([i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"Ram\"] = get_ram([i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"Ram\"] = get_ram([i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"Warranty\"] = get_warranty(df_train[\"Model_Info\"].values)\n",
    "df_test[\"Warranty\"] = get_warranty(df_test[\"Model_Info\"].values)\n",
    "\n",
    "df_train[\"Cash\"] = get_cash(df_train[\"Model_Info\"].values)\n",
    "df_test[\"Cash\"] = get_cash(df_test[\"Model_Info\"].values)\n",
    "\n",
    "df_train[\"iphone_type\"], df_train[\"iwatch_type\"] = get_apple_product(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"iphone_type\"], df_test[\"iwatch_type\"] = get_apple_product(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"LG_type\"] = get_lg_product(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"LG_type\"] = get_lg_product(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"Honor_type\"] = get_honor_product(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"Honor_type\"] = get_honor_product(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"Lenovo_type\"] = get_lenovo_product(df_train[\"Brand\"].values, [i.lower().split() for i in df_train[\"Model_Info\"].values])\n",
    "df_test[\"Lenovo_type\"] = get_lenovo_product(df_test[\"Brand\"].values, [i.lower().split() for i in df_test[\"Model_Info\"].values])\n",
    "\n",
    "df_train[\"Bad_condition\"] = get_bad_condition(df_train[\"Model_Info\"].values)\n",
    "df_test[\"Bad_condition\"] = get_bad_condition(df_test[\"Model_Info\"].values)\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert to numpy arrays and build the training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2326, 595) (997, 595) (2326,)\n"
     ]
    }
   ],
   "source": [
    "# For sparse, count based vectorization of model_info column\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 2), min_df=6)\n",
    "# One-hot-encoder for the brand column\n",
    "brand_ohe = OneHotEncoder(sparse=False, drop=\"first\")\n",
    "\n",
    "# Each feature is extracted individually from the pandas dataframe\n",
    "# and converted into a matrix form for easier concatenation\n",
    "\n",
    "# ------------------------------------- THIS IS TRAIN -------------------------------------\n",
    "brand_train = brand_ohe.fit_transform(np.reshape(df_train[\"Brand\"].values, (-1, 1)))\n",
    "rom_train = np.reshape(df_train[\"Rom\"].values, (-1, 1))\n",
    "ram_train = np.reshape(df_train[\"Ram\"].values, (-1, 1))\n",
    "warranty_train = np.reshape(df_train[\"Warranty\"].values, (-1, 1))\n",
    "cash_train = np.reshape(df_train[\"Cash\"].values, (-1, 1))\n",
    "iphone_type_train = np.reshape(df_train[\"iphone_type\"].values, (-1, 1))\n",
    "lg_type_train = np.reshape(df_train[\"LG_type\"].values, (-1, 1))\n",
    "honor_type_train = np.reshape(df_train[\"Honor_type\"].values, (-1, 1))\n",
    "lenovo_type_train = np.reshape(df_train[\"Lenovo_type\"].values, (-1, 1))\n",
    "bad_cond_train = np.reshape(df_train[\"Bad_condition\"].values, (-1, 1))\n",
    "product_train = df_train[\"Model_Info\"].values\n",
    "l_train = np.reshape(df_train[\"Locality\"].values, (-1, 1))\n",
    "sent_vecs_train = load_sent_vecs(\"train_sents.bin\")\n",
    "X_train = np.concatenate((brand_train, rom_train, warranty_train, cash_train, iphone_type_train, lg_type_train, honor_type_train, lenovo_type_train, bad_cond_train, vectorizer.fit_transform(product_train).toarray(), l_train, sent_vecs_train), axis=1)\n",
    "\n",
    "# ------------------------------------- THIS IS TEST -------------------------------------\n",
    "brand_test = brand_ohe.transform(np.reshape(df_test[\"Brand\"].values, (-1, 1)))\n",
    "rom_test = np.reshape(df_test[\"Rom\"].values, (-1, 1))\n",
    "ram_test = np.reshape(df_test[\"Ram\"].values, (-1, 1))\n",
    "warranty_test = np.reshape(df_test[\"Warranty\"].values, (-1, 1))\n",
    "cash_test = np.reshape(df_test[\"Cash\"].values, (-1, 1))\n",
    "iphone_type_test = np.reshape(df_test[\"iphone_type\"].values, (-1, 1))\n",
    "lg_type_test = np.reshape(df_test[\"LG_type\"].values, (-1, 1))\n",
    "honor_type_test = np.reshape(df_test[\"Honor_type\"].values, (-1, 1))\n",
    "lenovo_type_test = np.reshape(df_test[\"Lenovo_type\"].values, (-1, 1))\n",
    "bad_cond_test = np.reshape(df_test[\"Bad_condition\"].values, (-1, 1))\n",
    "product_test = df_test[\"Model_Info\"].values\n",
    "l_test = np.reshape(df_test[\"Locality\"].values, (-1, 1))\n",
    "sent_vecs_test = load_sent_vecs(\"test_sents.bin\")\n",
    "X_test = np.concatenate((brand_test, rom_test, warranty_test, cash_test, iphone_type_test, lg_type_test, honor_type_test, lenovo_type_test, bad_cond_test, vectorizer.transform(product_test).toarray(), l_test, sent_vecs_test), axis=1)\n",
    "\n",
    "# Target Variable\n",
    "Y = df_train[\"Price\"].values\n",
    "# This performs binning of the target variable\n",
    "# which defines a set of \"classes\" for the dataset\n",
    "Y_classes = [math.ceil(i/20000)-1 for i in df_train[\"Price\"].values]\n",
    "\n",
    "print(X_train.shape, X_test.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-Validation 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.042197\n",
      "0:\tlearn: 0.8197366\ttotal: 64.8ms\tremaining: 1m 4s\n",
      "500:\tlearn: 0.2305250\ttotal: 6.44s\tremaining: 6.41s\n",
      "999:\tlearn: 0.1359496\ttotal: 13.2s\tremaining: 0us\n",
      "0.41487648263683274\n",
      "Learning rate set to 0.042202\n",
      "0:\tlearn: 0.8300776\ttotal: 21.2ms\tremaining: 21.1s\n",
      "500:\tlearn: 0.2227492\ttotal: 6.98s\tremaining: 6.95s\n",
      "999:\tlearn: 0.1341088\ttotal: 14.1s\tremaining: 0us\n",
      "0.4334459475821377\n",
      "Learning rate set to 0.042202\n",
      "0:\tlearn: 0.8074907\ttotal: 16ms\tremaining: 16s\n",
      "500:\tlearn: 0.2286893\ttotal: 6.64s\tremaining: 6.62s\n",
      "999:\tlearn: 0.1362247\ttotal: 13.1s\tremaining: 0us\n",
      "0.4626448421603061\n",
      "Average:  0.4369890907930922\n"
     ]
    }
   ],
   "source": [
    "# I have created two sets of cross-validations in order to make sure that I am not overfitting\n",
    "# The classes I generated above is used to create a stratified sampling based cross-validation\n",
    "# This has been done in order to make both cross-validations as different as possible\n",
    "kfold, scores = StratifiedKFold(n_splits=3, shuffle=True, random_state=27), list()\n",
    "for train, test in kfold.split(X_train, Y_classes):\n",
    "    x_train, x_test = X_train[train], X_train[test]\n",
    "    y_train, y_test = np.log(Y[train]), Y[test]\n",
    "    \n",
    "    model = CatBoostRegressor(random_state=27, verbose=500)\n",
    "    model.fit(x_train, y_train)\n",
    "    preds1 = np.exp(model.predict(x_test))\n",
    "    \n",
    "    model = XGBRegressor(random_state=27, n_jobs=-1, objective=\"reg:squarederror\", max_depth=6, n_estimators=100)\n",
    "    model.fit(x_train, y_train)\n",
    "    preds2 = np.exp(model.predict(x_test))\n",
    "    \n",
    "    # Perform weighted average\n",
    "    preds = list()\n",
    "    catb, xgb = 0.7, 0.3\n",
    "    for a, b in zip(preds1, preds2):\n",
    "        preds.append(a*catb + b*xgb)\n",
    "    \n",
    "    score = np.sqrt(mean_squared_log_error(y_test, preds))\n",
    "    print(score)\n",
    "    scores.append(score)\n",
    "print(\"Average: \", sum(scores)/len(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross-Validation 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8153081\ttotal: 16.4ms\tremaining: 16.4s\n",
      "500:\tlearn: 0.2487873\ttotal: 6.78s\tremaining: 6.75s\n",
      "999:\tlearn: 0.1593901\ttotal: 13.6s\tremaining: 0us\n",
      "0.41733904165247115\n",
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8169781\ttotal: 16.9ms\tremaining: 16.9s\n",
      "500:\tlearn: 0.2357512\ttotal: 6.97s\tremaining: 6.94s\n",
      "999:\tlearn: 0.1420599\ttotal: 14.3s\tremaining: 0us\n",
      "0.47533243654977547\n",
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8232443\ttotal: 16.2ms\tremaining: 16.2s\n",
      "500:\tlearn: 0.2552304\ttotal: 6.85s\tremaining: 6.82s\n",
      "999:\tlearn: 0.1628011\ttotal: 13.6s\tremaining: 0us\n",
      "0.3928996830988315\n",
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8175939\ttotal: 16.4ms\tremaining: 16.4s\n",
      "500:\tlearn: 0.2448536\ttotal: 6.92s\tremaining: 6.89s\n",
      "999:\tlearn: 0.1552102\ttotal: 13.9s\tremaining: 0us\n",
      "0.4184548692175226\n",
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8174288\ttotal: 16.1ms\tremaining: 16.1s\n",
      "500:\tlearn: 0.2465476\ttotal: 6.79s\tremaining: 6.76s\n",
      "999:\tlearn: 0.1547253\ttotal: 13.6s\tremaining: 0us\n",
      "0.4703099432441624\n",
      "Learning rate set to 0.044514\n",
      "0:\tlearn: 0.8179693\ttotal: 16ms\tremaining: 16s\n",
      "500:\tlearn: 0.2435359\ttotal: 7.45s\tremaining: 7.42s\n",
      "999:\tlearn: 0.1537442\ttotal: 14.3s\tremaining: 0us\n",
      "0.3790981785488711\n",
      "Learning rate set to 0.044518\n",
      "0:\tlearn: 0.8186497\ttotal: 17.4ms\tremaining: 17.4s\n",
      "500:\tlearn: 0.2418662\ttotal: 6.79s\tremaining: 6.77s\n",
      "999:\tlearn: 0.1471579\ttotal: 14.1s\tremaining: 0us\n",
      "0.43156302963234083\n",
      "Learning rate set to 0.044518\n",
      "0:\tlearn: 0.8218078\ttotal: 16.1ms\tremaining: 16.1s\n",
      "500:\tlearn: 0.2423920\ttotal: 6.86s\tremaining: 6.83s\n",
      "999:\tlearn: 0.1551893\ttotal: 13.7s\tremaining: 0us\n",
      "0.4073994976613894\n",
      "Learning rate set to 0.044518\n",
      "0:\tlearn: 0.8216156\ttotal: 16.4ms\tremaining: 16.4s\n",
      "500:\tlearn: 0.2502898\ttotal: 6.8s\tremaining: 6.77s\n",
      "999:\tlearn: 0.1567804\ttotal: 13.6s\tremaining: 0us\n",
      "0.3827068746432274\n",
      "Learning rate set to 0.044518\n",
      "0:\tlearn: 0.8183564\ttotal: 16.1ms\tremaining: 16.1s\n",
      "500:\tlearn: 0.2334974\ttotal: 6.94s\tremaining: 6.91s\n",
      "999:\tlearn: 0.1451030\ttotal: 13.8s\tremaining: 0us\n",
      "0.4748653743396566\n",
      "Average:  0.4249968928588248\n"
     ]
    }
   ],
   "source": [
    "# This is simple validation\n",
    "kfold, scores = KFold(n_splits=10, shuffle=True, random_state=0), list()\n",
    "for train, test in kfold.split(X_train):\n",
    "    x_train, x_test = X_train[train], X_train[test]\n",
    "    y_train, y_test = np.log(Y[train]), Y[test]\n",
    "    \n",
    "    model = CatBoostRegressor(random_state=27, verbose=500)\n",
    "    model.fit(x_train, y_train)\n",
    "    preds1 = np.exp(model.predict(x_test))\n",
    "    \n",
    "    model = XGBRegressor(random_state=27, n_jobs=-1, objective=\"reg:squarederror\", max_depth=6, n_estimators=100)\n",
    "    model.fit(x_train, y_train)\n",
    "    preds2 = np.exp(model.predict(x_test))\n",
    "    \n",
    "    # Perform weighted average\n",
    "    preds = list()\n",
    "    catb, xgb = 0.7, 0.3\n",
    "    for a, b in zip(preds1, preds2):\n",
    "        preds.append(a*catb + b*xgb)\n",
    "    \n",
    "    score = np.sqrt(mean_squared_log_error(y_test, preds))\n",
    "    print(score)\n",
    "    scores.append(score)\n",
    "print(\"Average: \", sum(scores)/len(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Catboost() CV1: 0.44047614405448604  CV2: 0.4300283443371219  LB: 0.42132\n",
    "# Catboost() CV1: 0.4374455744964137  CV2: 0.42790331067680276  LB: 0.42286\n",
    "# XGBoost() CV1: 0.4487763139480631  CV2: 0.43902423538318763  LB: Didn't submit\n",
    "\n",
    "# ------------------------------------- FINAL -------------------------------------\n",
    "\n",
    "# Catboost() (0.7) XGBoost(max_depth=6) (0.3) -> CV1: 0.43460516637991997  CV2: 0.4259029100829695  LB: 0.41392"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.045358\n",
      "0:\tlearn: 0.8184066\ttotal: 16.3ms\tremaining: 16.3s\n",
      "500:\tlearn: 0.2533859\ttotal: 6.97s\tremaining: 6.95s\n",
      "999:\tlearn: 0.1609349\ttotal: 14s\tremaining: 0us\n"
     ]
    }
   ],
   "source": [
    "model = CatBoostRegressor(random_state=27, verbose=500)\n",
    "model.fit(X_train, np.log(Y))\n",
    "preds1 = np.exp(model.predict(X_test))\n",
    "\n",
    "model = XGBRegressor(random_state=27, n_jobs=-1, objective=\"reg:squarederror\", max_depth=6, n_estimators=100)\n",
    "model.fit(X_train, np.log(Y))\n",
    "preds2 = np.exp(model.predict(X_test))\n",
    "\n",
    "preds = list()\n",
    "catb, xgb = 0.7, 0.3\n",
    "for a, b in zip(preds1, preds2):\n",
    "    preds.append(a*catb + b*xgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make final submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submit = pd.DataFrame({'Price': preds})\n",
    "df_submit.to_excel(\"submit.xlsx\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
